#!/bin/bash
###
 # @file tracing/test/test.sh
 #
 # @license
 # Copyright (c) 2009-2018
 # Fraunhofer Institute for Algorithms and Scientific Computing SCAI
 # for Fraunhofer-Gesellschaft
 #
 # This file is part of the SCAI framework LAMA.
 #
 # LAMA is free software: you can redistribute it and/or modify it under the
 # terms of the GNU Lesser General Public License as published by the Free
 # Software Foundation, either version 3 of the License, or (at your option)
 # any later version.
 #
 # LAMA is distributed in the hope that it will be useful, but WITHOUT ANY
 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 # FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for
 # more details.
 #
 # You should have received a copy of the GNU Lesser General Public License
 # along with LAMA. If not, see <http://www.gnu.org/licenses/>.
 # @endlicense
 #
 # @brief Tests for SCAI tracing
 # @author Jan Ecker
 # @date 03.09.2015
###

if ((BASH_VERSINFO[0] < 4))
then
        echo "For testing tracing you need bash version 4 or newer"
        exit
fi

# =====================================================================================================================
# Basic variables and functions definitions
#
# =====================================================================================================================

# set error variable
errors=0

# common regex pattern which matches the end of timing lins in .time files
genericTimePattern=", inclusive = "[0-9]{1,5}\.[0-9]{4,6}", exclusive = "[0-9]{1,5}\.[0-9]{4,6}$

# will be configured: if the test is executed without OpenMP numThreads should be set to 1
numThreads=@NUM_THREADS_TEST@

# we should use 4 threads for all tests
export OMP_NUM_THREADS=$numThreads

SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd $SCRIPTDIR

# =====================================================================================================================
# Function that prepares a new test case. It removes all existing .ct and .time files in the folder, sets the
# SCAI_TRACE environmental variable properly and executes the simpleTracingXXX program
# usage:    prepareTestCase executable $SCAI_TRACE_OPTIONS
# examples: prepareTestCase executable ct:time
function prepareTestCase {
    if [ $# -ne 2 ]; then
        echo "Invalid number of parameters: prepareTestCase <executable> <trace_options> !"
        exit 1
    fi

    # clean old tracing files
    rm -rf *.ct* *.time*
    
    # export SCAI_TRACE setting
    if [ -z "$2" ]; then
        unset SCAI_TRACE
    else
        export SCAI_TRACE=$2
    fi
   
    # execute simpleTracingXXX and check for errors
    $1 &> /dev/null
    if [ $? -ne 0 ]; then
        echo "Error while runtime execution!"
        errors=$(($errors + 1))
        ret=1
    else
        ret=0
    fi
    
    if [ -z "$2" ]; then
        echo "+ Running tests with unset SCAI_TRACE"
    else
        echo "+ Running tests with SCAI_TRACE=$2"
    fi
    
}

# =====================================================================================================================
# Function that checks the number of existing .ct files. Returns an error if number of existing files
# does not match $num
# usage:    checkCTFilesExist $num
# example:  checkCTFilesExist 1
function checkCTFilesExist {
    if [ $# -ne 1 ]; then
        echo "Invalid number of parameters!"
        exit 1
    fi
    
    count=`ls -l -la *.ct* 2> /dev/null | wc -l`
    if [ "$count" -ne "$1" ]; then
        echo "Test failed. There are $count *.ct files but it should be $1".    
        errors=$(($errors + 1))
        ret=1
    else
        ret=0
    fi
}

# =====================================================================================================================
# Function that checks the number of existing .time files. Returns an error if number of existing files
# does not match $num
# usage:    checkCTFilesExist $num
# example:  checkCTFilesExist 1
function checkTimeFilesExist {
    if [ $# -ne 1 ]; then
        echo "Invalid number of parameters!"
        exit 1
    fi
    
    count=`ls -l -la *.time* 2> /dev/null | wc -l`
    if [ "$count" -ne "$1" ]; then
        echo "Test failed. There are $count *.time files but it should be $1".    
        errors=$(($errors + 1))
        ret=1
    else
        ret=0
    fi
}

# =====================================================================================================================
# Function that checks the contents of the given .time file.
# usage:    checkTimeFileContents $FILE $NTHREADS
# example:  checkTimeFileContents simpleTracingON.time 4
function checkTimeFileContents {
    # all the regions with the correct number of calls should appear in the .time file

    if [ $# -ne 2 ]; then
        echo "Invalid number of parameters!"
        exit 1
    fi
    
    nThreads=$2

    content=`cat $1 2> /dev/null`
    
    # check for region 'Time main'
    # the main method is not executed in parallel and therefore should only match once!
    count=`echo "$content" | grep -E "^Time main \(in ms\) : #calls = 1${genericTimePattern}" | wc -l`
    if [ "$count" -ne 1 ]; then
        echo "ERROR: Content of the .time file is wrong (region main)"
        errors=$(($errors + 1))
    fi
            
    # check for region 'Time A'
    numCalls=$((300000/$numThreads))
    count=`echo "$content" | grep -E "^Time A \(in ms\) : #calls = ${numCalls}${genericTimePattern}" | wc -l`
    if [ "$count" -ne "$nThreads" ]; then
        echo "ERROR: Content of the .time file is wrong (region A)"
        errors=$(($errors + 1))
    fi
    
    # check for region 'Time B'
    numCalls=$((200000/$numThreads))
    count=`echo "$content" | grep -E "^Time B \(in ms\) : #calls = ${numCalls}${genericTimePattern}" | wc -l`
    if [ "$count" -ne "$nThreads" ]; then
        echo "ERROR: Content of the .time file is wrong (region B)"
        errors=$(($errors + 1))
    fi
    
    # check for region 'Time main.loopA'
    numCalls=$((10000/$numThreads))
    count=`echo "$content" | grep -E "^Time main.loopA \(in ms\) : #calls = ${numCalls}${genericTimePattern}" | wc -l`
    if [ "$count" -ne "$nThreads" ]; then
        echo "ERROR: Content of the .time file is wrong (region main.loopA)"
        errors=$(($errors + 1))
    fi
    
    # check for region 'Time main.loopB'
    numCalls=$((10000/$numThreads))
    count=`echo "$content" | grep -E "^Time main.loopB \(in ms\) : #calls = ${numCalls}${genericTimePattern}" | wc -l`
    if [ "$count" -ne "$nThreads" ]; then
        echo "ERROR: Content of the .time file is wrong (region main.loopB)"
        errors=$(($errors + 1))
    fi
    
    # the file should contain 3 header lines and 4 "timing" lines for each thread and one addition line for the main
    # function (in the main thread)
    expectedLines=$(($nThreads * 7 + 1))
    
    # check if the file contains more then that
    lines=`echo "$content" | wc -l`
    if [ "$lines" -ne "$expectedLines" ]; then
        echo "ERROR: The .time file contains unknown content"
        errors=$(($errors + 1))
    fi
}

# =====================================================================================================================
# Function that checks the contents of the given .ct file. It looks for all files that have the syntax $FILE.*
# if $NTHREADS is bigger 1
# usage:    checkCTFileContents $FILE $NTHREADS
# example:  checkCTFileContents simpleTracingON.ct 4

function checkCTFileContents {
    # the structure of the .ct files is quite complicated and can't be fully tested here. We therefore only do
    # some quick validity checks
    
    if [ $# -ne 2 ]; then
        echo "Invalid number of parameters!"
        exit 1
    fi
    
    nThreads=$2

    # The following loop checks all the criteria that should be matched by ALL .ct files
    for file in ${1}*; do
        content=`cat $file 2> /dev/null`
        
        # check for region 'loopA'
        output=`echo "$content" | grep -E '^fn [0-9]+ [0-9]+ loopA [0-9]+ [0-9]+ [0-9]+ main$'`
        if [ $? -ne 0 ]; then
            echo "ERROR: Content of the .ct file is wrong (region loopA)"
            errors=$(($errors + 1))
        fi
    
        # check for region 'loopB'
        output=`echo "$content" | grep -E '^fn [0-9]+ [0-9]+ loopB [0-9]+ [0-9]+ [0-9]+ main$'`
        if [ $? -ne 0 ]; then
            echo "ERROR: Content of the .ct file is wrong (region loopB)"
            errors=$(($errors + 1))
        fi
        
        # check for region 'A'
        output=`echo "$content" | grep -E '^fn [0-9]+ [0-9]+ A [0-9]+ [0-9]+ [0-9]+ \?$'`
        if [ $? -ne 0 ]; then
            echo "ERROR: Content of the .ct file is wrong (region A)"
            errors=$(($errors + 1))
        fi
        
        # check for region 'B'
        output=`echo "$content" | grep -E '^fn [0-9]+ [0-9]+ B [0-9]+ [0-9]+ [0-9]+ \?$'`
        if [ $? -ne 0 ]; then
            echo "ERROR: Content of the .ct file is wrong (region B)"
            errors=$(($errors + 1))
        fi
        
        
        # we can also check if there is a line containing the correct number of lines
        
        # check for region 'A'
        numCalls=$((300000/$numThreads))
        output=`echo "$content" | grep -E "^calls ${numCalls} 0$"`
        if [ $? -ne 0 ]; then
            echo "ERROR: Content of the .ct file is wrong (calls region A)"
            errors=$(($errors + 1))
        fi
        
        # check for region 'B'
        numCalls=$((200000/$numThreads))
        output=`echo "$content" | grep -E "^calls ${numCalls} 0$"`
        if [ $? -ne 0 ]; then
            echo "ERROR: Content of the .ct file is wrong (calls region B)"
            errors=$(($errors + 1))
        fi
        
    done
    
    
    
    # There is some content that can only be found in the "main" .ct file as the parallism is created in the main
    # function of the program. The following loop checks whether there is exactly one file that matches the related
    # criteria
    
    found=0
    for file in ${1}*; do
        content=`cat $file 2> /dev/null`
    
        # The "main" .ct file should contain the 'main' region
        output=`echo "$content" | grep -E '^fn [0-9]+ [0-9]+ main [0-9]+ [0-9]+ [0-9]+ \?$'`
        if [ $? -ne 0 ]; then
            continue
            echo "ERROR: Content of the .ct file is wrong (region main)"
            errors=$(($errors + 1))
        fi
        
        # Only the "main" .ct file contains the following call numbers of 'loopA' and 'loopB'
        numCalls=$((10000/$numThreads))
        output=`echo "$content" | grep -E "^calls ${numCalls} 0$" | wc -l`
        if [ "$output" -ne 2 ]; then
            continue
            echo "ERROR: Content of the .ct file is wrong (calls region loopA / loopB)"
            errors=$(($errors + 1))
        fi
    
        # check for some other structural properties
        
        # check if there are exactly 4 call cost regions
        output=`echo "$content" | grep 'begin call cost line' | wc -l`
        if [ "$output" -ne 4 ]; then
            continue
            echo "ERROR: Content of the .ct file is wrong (number of call cost regions)"
            errors=$(($errors + 1))
        fi
        
        # check if there are exactly 4 exclusive call cost regions
        output=`echo "$content" | grep 'begin exclusive cost line' | wc -l`
        if [ "$output" -ne 5 ]; then
            continue
            echo "ERROR: Content of the .ct file is wrong (number of exclusive call cost regions)"
            errors=$(($errors + 1))
        fi
        
        if [ $found -eq 1 ]; then
            echo "ERROR: Multiple .ct file matched the criteria for the "main" file containing the full calltree!"
            errors=$(($errors + 1))
        fi
        found=1
    done

    if [ "$found" -eq 0 ]; then
        echo "ERROR: No .ct file matched the criteria for the "main" file containing the full calltree!"
        errors=$(($errors + 1))
    fi
}

# =====================================================================================================================
# Runtime configurations tests
#
# In this test the executable is build WITH trace support and the runtime configuration via environmental variables is
# used to control the tracing behavior. 
# =====================================================================================================================

echo "Running runtime configuration tests (simpleTracingON) :"

# =================================================================================================================
# Test 1
# check execution with unset SCAI_TRACE
    
prepareTestCase ./simpleTracingON ""
if [ $ret -eq 0 ]; then
    # If tracing is disabled, NO tracing files should be created, so we check if *.ct or *.time files exist
    checkCTFilesExist 0
    checkTimeFilesExist 0
fi
    
# =================================================================================================================
# Test 2
# check execution with SCAI_TRAC=time

prepareTestCase ./simpleTracingON time
if [ $ret -eq 0 ]; then

    checkCTFilesExist 0
    
    checkTimeFilesExist 1
    if [ $ret -eq 0 ]; then
        checkTimeFileContents simpleTracingON.time 1
    fi
fi
    
# =================================================================================================================
# Test 3
# check execution with SCAI_TRACE=ct

prepareTestCase ./simpleTracingON ct

if [ $ret -eq 0 ]; then
    # there should be .time files
    checkTimeFilesExist 0
    
    checkCTFilesExist 1
    if [ $ret -eq 0 ]; then
        checkCTFileContents simpleTracingON.ct 1
    fi
fi
        
# =================================================================================================================
# Test 4
# check execution with SCAI_TRACE=time:PREFIX=customPrefix

prepareTestCase ./simpleTracingON time:PREFIX=customPrefix
    
if [ $ret -eq 0 ]; then

    checkCTFilesExist 0
    
    # we have to check if there is exactly one .time file (no additional .time files created)
    checkTimeFilesExist 1
        
        
    # Check whether the a correct .time file was generated
    count=`ls -l -la customPrefix.time 2> /dev/null | wc -l`
    if [ $count -ne 1 ]; then
        echo "Test failed. No .time file has been generated or a wrong name was used."
        errors=$(($errors + 1))
    else
        checkTimeFileContents customPrefix.time 1
    fi
fi
    
# =================================================================================================================
# Test 5
# check execution with SCAI_TRACE=ct:time

prepareTestCase ./simpleTracingON ct:time

if [ $ret -eq 0 ]; then
    checkCTFilesExist 1
    
    if [ $ret -eq 0 ]; then
        checkCTFileContents simpleTracingON.ct 1
    fi
    
    checkTimeFilesExist 1
    
    if [ $ret -eq 0 ]; then
        checkTimeFileContents simpleTracingON.time 1
    fi
fi
    
    
# =================================================================================================================    
# Test 6
# check execution with SCAI_TRACE=time:thread
    
prepareTestCase ./simpleTracingON time:thread
    
if [ $ret -eq 0 ]; then

    # there should be no .ct files 

    checkCTFilesExist 0
    
    checkTimeFilesExist 1
    if [ $ret -eq 0 ]; then
        checkTimeFileContents simpleTracingON.time $numThreads
    fi
fi
    
# =================================================================================================================
# Test 7
# check execution with SCAI_TRACE=ct:thread
    
prepareTestCase ./simpleTracingON ct:thread
    
if [ $ret -eq 0 ]; then
    # there should be no .time files
    checkTimeFilesExist 0
    
    checkCTFilesExist $numThreads
    if [ $ret -eq 0 ]; then
        checkCTFileContents simpleTracingON.ct $numThreads
    fi
fi

# =================================================================================================================
# Test 8
# check execution width SCAI_TRACE=time:ct:thread

prepareTestCase ./simpleTracingON time:ct:thread
    
if [ $ret -eq 0 ]; then
    checkCTFilesExist $numThreads
    if [ $ret -eq 0 ]; then
        checkCTFileContents simpleTracingON.ct $numThreads
    fi
    
    checkTimeFilesExist 1
    if [ $ret -eq 0 ]; then
        checkTimeFileContents simpleTracingON.time $numThreads
    fi
fi
    
# =================================================================================================================
# Test 9
# check execution with SCAI_TRACE=PREFIX=customPrefix:time:ct:thread

prepareTestCase ./simpleTracingON PREFIX=customPrefix:time:ct:thread

if [ $ret -eq 0 ]; then
    checkCTFilesExist $numThreads

    # Check wheter the correct naming was used        
    count=`ls -l -la customPrefix.ct.* 2> /dev/null | wc -l`
    if [ $count -ne $numThreads ]; then
        echo "Test failed. Wrong number of .ct file has been generated or a wrong names were used."
        errors=$(($errors + 1))
    else
        checkCTFileContents customPrefix.ct $numThreads
    fi


    checkTimeFilesExist 1
    
    # Check whether the a correct .time file was generated
    count=`ls -l -la customPrefix.time 2> /dev/null | wc -l`
    if [ $count -ne 1 ]; then
        echo "Test failed. No .time file has been generated or a wrong name was used."
        errors=$(($errors + 1))
    else
        checkTimeFileContents customPrefix.time $numThreads
    fi
fi

# =====================================================================================================================
# Compile time tests
#
# =====================================================================================================================

# Check that no tracing files are generated if executable is built without trace support but SCAI_TRACE is set   

echo ""
echo "Running compile time tests:"

prepareTestCase ./simpleTracingOFF ct:time
    
# There should be no .ct or .time files
checkCTFilesExist 0
checkTimeFilesExist 0

prepareTestCase ./simpleTracingON0 ct:time:thread
    
checkCTFilesExist $numThreads
if [ $ret -eq 0 ]; then
    checkCTFileContents ./simpleTracingON0.ct $numThreads
fi
    
checkTimeFilesExist 1
if [ $ret -eq 0 ]; then
    checkTimeFileContents ./simpleTracingON0.time $numThreads
fi


# =====================================================================================================================
# Check for errors and return with proper exist code
# =====================================================================================================================

echo ""
if [ $errors -eq 0 ]; then
    echo "Tests run sucessfully!"
    exit
else
    echo "Tests failed! Number of errors: $errors"
    exit 1
fi
